Node / ReticulumProjects / MeshChatX.git / files / tests / backend / test_performance_bottlenecks.py

Displaying Raw • Download

Tff7b72import T7ee787os

Tff7b72import T7ee787secrets

Tff7b72import T7ee787shutil

Tff7b72import T7ee787tempfile

Tff7b72import T7ee787time

Tff7b72import T7ee787unittest

Tff7b72from T7ee787unittestT7ee787.T7ee787mock Tff7b72import Te6edf3MagicMock

Tff7b72from T7ee787meshchatxT7ee787.T7ee787srcT7ee787.T7ee787backendT7ee787.T7ee787announce_manager Tff7b72import Te6edf3AnnounceManager

Tff7b72from T7ee787meshchatxT7ee787.T7ee787srcT7ee787.T7ee787backendT7ee787.T7ee787database Tff7b72import Te6edf3Database

Tff7b72class T56d364TestPerformanceBottlenecksTb4b4b4(Te6edf3unittestTff7b72.Td2a8ffTestCaseTb4b4b4)Tb4b4b4:

Tff7b72def Td2a8ffsetUpTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:

Tff7b72selfTff7b72.Td2a8fftest_dir Tff7b72= Te6edf3tempfileTff7b72.Td2a8ffmkdtempTb4b4b4(Tb4b4b4)

Tff7b72selfTff7b72.Td2a8ffdb_path Tff7b72= Te6edf3osTff7b72.Td2a8ffpathTff7b72.Td2a8ffjoinTb4b4b4(Tff7b72selfTff7b72.Td2a8fftest_dirTb4b4b4, Ta5d6ff"Ta5d6ffperf_bottleneck.dbTa5d6ff"Tb4b4b4)

Tff7b72selfTff7b72.Td2a8ffdb Tff7b72= Te6edf3DatabaseTb4b4b4(Tff7b72selfTff7b72.Td2a8ffdb_pathTb4b4b4)

Tff7b72selfTff7b72.Td2a8ffdbTff7b72.Td2a8ffinitializeTb4b4b4(Tb4b4b4)

Tff7b72selfTff7b72.Td2a8ffannounce_manager Tff7b72= Te6edf3AnnounceManagerTb4b4b4(Tff7b72selfTff7b72.Td2a8ffdbTb4b4b4)

Tff7b72selfTff7b72.Td2a8ffreticulum_mock Tff7b72= Te6edf3MagicMockTb4b4b4(Tb4b4b4)

Tff7b72selfTff7b72.Td2a8ffreticulum_mockTff7b72.Td2a8ffget_packet_rssiTff7b72.Td2a8ffreturn_value Tff7b72= Tff7b72-T79c0ff50

Tff7b72selfTff7b72.Td2a8ffreticulum_mockTff7b72.Td2a8ffget_packet_snrTff7b72.Td2a8ffreturn_value Tff7b72= T79c0ff5.0

Tff7b72selfTff7b72.Td2a8ffreticulum_mockTff7b72.Td2a8ffget_packet_qTff7b72.Td2a8ffreturn_value Tff7b72= T79c0ff3

Tff7b72def Td2a8fftearDownTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:

Tff7b72selfTff7b72.Td2a8ffdbTff7b72.Td2a8ffclose_allTb4b4b4(Tb4b4b4)

Te6edf3shutilTff7b72.Td2a8ffrmtreeTb4b4b4(Tff7b72selfTff7b72.Td2a8fftest_dirTb4b4b4)

Tff7b72def Td2a8fftest_message_pagination_performanceTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:

T8b949e"""Test performance of message pagination with a large dataset."""

Te6edf3num_messages Tff7b72= T79c0ff10000

Te6edf3peer_hash Tff7b72= Te6edf3secretsTff7b72.Td2a8fftoken_hexTb4b4b4(T79c0ff16Tb4b4b4)

Tffa657printTb4b4b4(Ta5d6fffTa5d6ff"Tffea00\nTa5d6ffSeeding Tffd700{Te6edf3num_messagesTffd700}Ta5d6ff messages for pagination test...Ta5d6ff"Tb4b4b4)

Tff7b72with Tff7b72selfTff7b72.Td2a8ffdbTff7b72.Td2a8ffproviderTb4b4b4:

Tff7b72for Te6edf3i Tff7b72in Tffa657rangeTb4b4b4(Te6edf3num_messagesTb4b4b4)Tb4b4b4:

Te6edf3msg Tff7b72= Tb4b4b4{

Ta5d6ff"Ta5d6ffhashTa5d6ff"Tb4b4b4: Te6edf3secretsTff7b72.Td2a8fftoken_hexTb4b4b4(T79c0ff16Tb4b4b4)Tb4b4b4,

Ta5d6ff"Ta5d6ffsource_hashTa5d6ff"Tb4b4b4: Te6edf3peer_hashTb4b4b4,

Ta5d6ff"Ta5d6ffdestination_hashTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffmy_hashTa5d6ff"Tb4b4b4,

Ta5d6ff"Ta5d6ffpeer_hashTa5d6ff"Tb4b4b4: Te6edf3peer_hashTb4b4b4,

Ta5d6ff"Ta5d6ffstateTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffdeliveredTa5d6ff"Tb4b4b4,

Ta5d6ff"Ta5d6ffprogressTa5d6ff"Tb4b4b4: T79c0ff1.0Tb4b4b4,

Ta5d6ff"Ta5d6ffis_incomingTa5d6ff"Tb4b4b4: T79c0ff1Tb4b4b4,

Ta5d6ff"Ta5d6ffmethodTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffdirectTa5d6ff"Tb4b4b4,

Ta5d6ff"Ta5d6ffdelivery_attemptsTa5d6ff"Tb4b4b4: T79c0ff1Tb4b4b4,

Ta5d6ff"Ta5d6fftitleTa5d6ff"Tb4b4b4: Ta5d6fffTa5d6ff"Ta5d6ffMsg Tffd700{Te6edf3iTffd700}Ta5d6ff"Tb4b4b4,

Ta5d6ff"Ta5d6ffcontentTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffContentTa5d6ff"Tb4b4b4,

Ta5d6ff"Ta5d6fffieldsTa5d6ff"Tb4b4b4: Ta5d6ff"Tffd700{}Ta5d6ff"Tb4b4b4,

Ta5d6ff"Ta5d6fftimestampTa5d6ff"Tb4b4b4: Te6edf3timeTff7b72.Td2a8fftimeTb4b4b4(Tb4b4b4) Tff7b72- Te6edf3iTb4b4b4,

Ta5d6ff"Ta5d6ffrssiTa5d6ff"Tb4b4b4: Tff7b72-T79c0ff50Tb4b4b4,

Ta5d6ff"Ta5d6ffsnrTa5d6ff"Tb4b4b4: T79c0ff5.0Tb4b4b4,

Ta5d6ff"Ta5d6ffqualityTa5d6ff"Tb4b4b4: T79c0ff3Tb4b4b4,

Ta5d6ff"Ta5d6ffis_spamTa5d6ff"Tb4b4b4: T79c0ff0Tb4b4b4,

Tb4b4b4}

Tff7b72selfTff7b72.Td2a8ffdbTff7b72.Td2a8ffmessagesTff7b72.Td2a8ffupsert_lxmf_messageTb4b4b4(Te6edf3msgTb4b4b4)

T8b949e# Benchmark different offsets

Te6edf3offsets Tff7b72= Tb4b4b4[T79c0ff0Tb4b4b4, T79c0ff1000Tb4b4b4, T79c0ff5000Tb4b4b4, T79c0ff9000Tb4b4b4]

Te6edf3limit Tff7b72= T79c0ff50

Tff7b72for Te6edf3offset Tff7b72in Te6edf3offsetsTb4b4b4:

Te6edf3start Tff7b72= Te6edf3timeTff7b72.Td2a8fftimeTb4b4b4(Tb4b4b4)

Te6edf3msgs Tff7b72= Tff7b72selfTff7b72.Td2a8ffdbTff7b72.Td2a8ffmessagesTff7b72.Td2a8ffget_conversation_messagesTb4b4b4(

Te6edf3peer_hashTb4b4b4,

Te6edf3limitTff7b72=Te6edf3limitTb4b4b4,

Te6edf3offsetTff7b72=Te6edf3offsetTb4b4b4,

Tb4b4b4)

Te6edf3duration Tff7b72= Tb4b4b4(Te6edf3timeTff7b72.Td2a8fftimeTb4b4b4(Tb4b4b4) Tff7b72- Te6edf3startTb4b4b4) Tff7b72* T79c0ff1000

Tffa657printTb4b4b4(Ta5d6fffTa5d6ff"Ta5d6ffFetch Tffd700{Te6edf3limitTffd700}Ta5d6ff messages at offset Tffd700{Te6edf3offsetTffd700}Ta5d6ff: Tffd700{Te6edf3durationTffd700:Ta5d6ff.2fTffd700}Ta5d6ffmsTa5d6ff"Tb4b4b4)

Tff7b72selfTff7b72.Td2a8ffassertEqualTb4b4b4(Tffa657lenTb4b4b4(Te6edf3msgsTb4b4b4)Tb4b4b4, Te6edf3limitTb4b4b4)

Tff7b72selfTff7b72.Td2a8ffassertLessTb4b4b4(Te6edf3durationTb4b4b4, T79c0ff50Tb4b4b4, Ta5d6fffTa5d6ff"Ta5d6ffPagination at offset Tffd700{Te6edf3offsetTffd700}Ta5d6ff is too slow!Ta5d6ff"Tb4b4b4)

Tff7b72def Td2a8fftest_announce_flood_bottleneckTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:

T8b949e"""Simulate a flood of incoming announces and measure processing time."""

Te6edf3num_announces Tff7b72= T79c0ff500

Te6edf3identities Tff7b72= Tb4b4b4[Te6edf3MagicMockTb4b4b4(Tb4b4b4) Tff7b72for Te6edf3_ Tff7b72in Tffa657rangeTb4b4b4(Te6edf3num_announcesTb4b4b4)Tb4b4b4]

Tff7b72for Te6edf3iTb4b4b4, Te6edf3ident Tff7b72in Tffa657enumerateTb4b4b4(Te6edf3identitiesTb4b4b4)Tb4b4b4:

Te6edf3identTff7b72.Td2a8ffhash Tff7b72= Te6edf3MagicMockTb4b4b4(Tb4b4b4)

Te6edf3identTff7b72.Td2a8ffhashTff7b72.Td2a8ffhexTff7b72.Td2a8ffreturn_value Tff7b72= Te6edf3secretsTff7b72.Td2a8fftoken_hexTb4b4b4(T79c0ff16Tb4b4b4)

Te6edf3identTff7b72.Td2a8ffget_public_keyTff7b72.Td2a8ffreturn_value Tff7b72= Ta5d6ffbTa5d6ff"Ta5d6ffpublic_keyTa5d6ff"

Tffa657printTb4b4b4(Ta5d6fffTa5d6ff"Tffea00\nTa5d6ffSimulating flood of Tffd700{Te6edf3num_announcesTffd700}Ta5d6ff announces...Ta5d6ff"Tb4b4b4)

Te6edf3start_total Tff7b72= Te6edf3timeTff7b72.Td2a8fftimeTb4b4b4(Tb4b4b4)

T8b949e# We simulate what meshchat.on_lxmf_announce_received does

Tff7b72with Tff7b72selfTff7b72.Td2a8ffdbTff7b72.Td2a8ffproviderTb4b4b4:

Tff7b72for Te6edf3i Tff7b72in Tffa657rangeTb4b4b4(Te6edf3num_announcesTb4b4b4)Tb4b4b4:

Te6edf3dest_hash Tff7b72= Te6edf3secretsTff7b72.Td2a8fftoken_hexTb4b4b4(T79c0ff16Tb4b4b4)

Te6edf3aspect Tff7b72= Ta5d6ff"Ta5d6fflxmf.deliveryTa5d6ff"

Te6edf3app_data Tff7b72= Ta5d6ffbTa5d6ff"Ta5d6ffapp_dataTa5d6ff"

Te6edf3packet_hash Tff7b72= Ta5d6ffbTa5d6ff"Ta5d6ffpacket_hashTa5d6ff"

T8b949e# This is the synchronous part that could be a bottleneck

Tff7b72selfTff7b72.Td2a8ffannounce_managerTff7b72.Td2a8ffupsert_announceTb4b4b4(

Tff7b72selfTff7b72.Td2a8ffreticulum_mockTb4b4b4,

Te6edf3identitiesTb4b4b4[Te6edf3iTb4b4b4]Tb4b4b4,

Te6edf3dest_hashTb4b4b4,

Te6edf3aspectTb4b4b4,

Te6edf3app_dataTb4b4b4,

Te6edf3packet_hashTb4b4b4,

Tb4b4b4)

T8b949e# Simulate the fetch after upsert which is also done in the app

Tff7b72selfTff7b72.Td2a8ffdbTff7b72.Td2a8ffannouncesTff7b72.Td2a8ffget_announce_by_hashTb4b4b4(Te6edf3dest_hashTb4b4b4)

Te6edf3duration_total Tff7b72= Te6edf3timeTff7b72.Td2a8fftimeTb4b4b4(Tb4b4b4) Tff7b72- Te6edf3start_total

Te6edf3avg_duration Tff7b72= Tb4b4b4(Te6edf3duration_total Tff7b72/ Te6edf3num_announcesTb4b4b4) Tff7b72* T79c0ff1000

Tffa657printTb4b4b4(

Ta5d6fffTa5d6ff"Ta5d6ffProcessed Tffd700{Te6edf3num_announcesTffd700}Ta5d6ff announces in Tffd700{Te6edf3duration_totalTffd700:Ta5d6ff.2fTffd700}Ta5d6ffs (Avg: Tffd700{Te6edf3avg_durationTffd700:Ta5d6ff.2fTffd700}Ta5d6ffms/announce)Ta5d6ff"Tb4b4b4,

Tb4b4b4)

Tff7b72selfTff7b72.Td2a8ffassertLessTb4b4b4(Te6edf3avg_durationTb4b4b4, T79c0ff20Tb4b4b4, Ta5d6ff"Ta5d6ffAnnounce processing is too slow!Ta5d6ff"Tb4b4b4)

Tff7b72def Td2a8fftest_announce_pagination_performanceTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:

T8b949e"""Test performance of announce pagination with search and filtering."""

Te6edf3num_announces Tff7b72= T79c0ff5000

Tffa657printTb4b4b4(Ta5d6fffTa5d6ff"Tffea00\nTa5d6ffSeeding Tffd700{Te6edf3num_announcesTffd700}Ta5d6ff announces for pagination test...Ta5d6ff"Tb4b4b4)

Tff7b72with Tff7b72selfTff7b72.Td2a8ffdbTff7b72.Td2a8ffproviderTb4b4b4:

Tff7b72for Te6edf3i Tff7b72in Tffa657rangeTb4b4b4(Te6edf3num_announcesTb4b4b4)Tb4b4b4:

Te6edf3data Tff7b72= Tb4b4b4{

Ta5d6ff"Ta5d6ffdestination_hashTa5d6ff"Tb4b4b4: Te6edf3secretsTff7b72.Td2a8fftoken_hexTb4b4b4(T79c0ff16Tb4b4b4)Tb4b4b4,

Ta5d6ff"Ta5d6ffaspectTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6fflxmf.deliveryTa5d6ff" Tff7b72if Te6edf3i Tff7b72% T79c0ff2 Tff7b72== T79c0ff0 Tff7b72else Ta5d6ff"Ta5d6fflxst.telephonyTa5d6ff"Tb4b4b4,

Ta5d6ff"Ta5d6ffidentity_hashTa5d6ff"Tb4b4b4: Te6edf3secretsTff7b72.Td2a8fftoken_hexTb4b4b4(T79c0ff16Tb4b4b4)Tb4b4b4,

Ta5d6ff"Ta5d6ffidentity_public_keyTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffpubkeyTa5d6ff"Tb4b4b4,

Ta5d6ff"Ta5d6ffapp_dataTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffdataTa5d6ff"Tb4b4b4,

Ta5d6ff"Ta5d6ffrssiTa5d6ff"Tb4b4b4: Tff7b72-T79c0ff50Tb4b4b4,

Ta5d6ff"Ta5d6ffsnrTa5d6ff"Tb4b4b4: T79c0ff5.0Tb4b4b4,

Ta5d6ff"Ta5d6ffqualityTa5d6ff"Tb4b4b4: T79c0ff3Tb4b4b4,

Tb4b4b4}

Tff7b72selfTff7b72.Td2a8ffdbTff7b72.Td2a8ffannouncesTff7b72.Td2a8ffupsert_announceTb4b4b4(Te6edf3dataTb4b4b4)

T8b949e# Benchmark filtered search with pagination

Te6edf3start Tff7b72= Te6edf3timeTff7b72.Td2a8fftimeTb4b4b4(Tb4b4b4)

Te6edf3results Tff7b72= Tff7b72selfTff7b72.Td2a8ffannounce_managerTff7b72.Td2a8ffget_filtered_announcesTb4b4b4(

Te6edf3aspectTff7b72=Ta5d6ff"Ta5d6fflxmf.deliveryTa5d6ff"Tb4b4b4,

Te6edf3limitTff7b72=T79c0ff50Tb4b4b4,

Te6edf3offsetTff7b72=T79c0ff1000Tb4b4b4,

Tb4b4b4)

Te6edf3duration Tff7b72= Tb4b4b4(Te6edf3timeTff7b72.Td2a8fftimeTb4b4b4(Tb4b4b4) Tff7b72- Te6edf3startTb4b4b4) Tff7b72* T79c0ff1000

Tffa657printTb4b4b4(Ta5d6fffTa5d6ff"Ta5d6ffFiltered announce pagination (offset 1000): Tffd700{Te6edf3durationTffd700:Ta5d6ff.2fTffd700}Ta5d6ffmsTa5d6ff"Tb4b4b4)

Tff7b72selfTff7b72.Td2a8ffassertEqualTb4b4b4(Tffa657lenTb4b4b4(Te6edf3resultsTb4b4b4)Tb4b4b4, T79c0ff50Tb4b4b4)

Tff7b72selfTff7b72.Td2a8ffassertLessTb4b4b4(Te6edf3durationTb4b4b4, T79c0ff50Tb4b4b4, Ta5d6ff"Ta5d6ffAnnounce pagination is too slow!Ta5d6ff"Tb4b4b4)

Tff7b72def Td2a8fftest_concurrent_announce_handlingTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:

T8b949e"""Test how the database handles concurrent announce insertions from multiple threads."""

Tff7b72import T7ee787threading

Te6edf3num_threads Tff7b72= T79c0ff10

Te6edf3announces_per_thread Tff7b72= T79c0ff50

Tff7b72def Td2a8ffinsert_announcesTb4b4b4(Tb4b4b4)Tb4b4b4:

Tff7b72for Te6edf3_ Tff7b72in Tffa657rangeTb4b4b4(Te6edf3announces_per_threadTb4b4b4)Tb4b4b4:

Te6edf3dest_hash Tff7b72= Te6edf3secretsTff7b72.Td2a8fftoken_hexTb4b4b4(T79c0ff16Tb4b4b4)

Te6edf3ident Tff7b72= Te6edf3MagicMockTb4b4b4(Tb4b4b4)

Te6edf3identTff7b72.Td2a8ffhashTff7b72.Td2a8ffhexTff7b72.Td2a8ffreturn_value Tff7b72= Te6edf3secretsTff7b72.Td2a8fftoken_hexTb4b4b4(T79c0ff16Tb4b4b4)

Te6edf3identTff7b72.Td2a8ffget_public_keyTff7b72.Td2a8ffreturn_value Tff7b72= Ta5d6ffbTa5d6ff"Ta5d6ffpubkeyTa5d6ff"

Tff7b72selfTff7b72.Td2a8ffannounce_managerTff7b72.Td2a8ffupsert_announceTb4b4b4(

Tff7b72selfTff7b72.Td2a8ffreticulum_mockTb4b4b4,

Te6edf3identTb4b4b4,

Te6edf3dest_hashTb4b4b4,

Ta5d6ff"Ta5d6fflxmf.deliveryTa5d6ff"Tb4b4b4,

Ta5d6ffbTa5d6ff"Ta5d6ffdataTa5d6ff"Tb4b4b4,

Ta5d6ffbTa5d6ff"Ta5d6ffpacketTa5d6ff"Tb4b4b4,

Tb4b4b4)

Te6edf3threads Tff7b72= Tb4b4b4[

Te6edf3threadingTff7b72.Td2a8ffThreadTb4b4b4(Te6edf3targetTff7b72=Te6edf3insert_announcesTb4b4b4) Tff7b72for Te6edf3_ Tff7b72in Tffa657rangeTb4b4b4(Te6edf3num_threadsTb4b4b4)

Tb4b4b4]

Tffa657printTb4b4b4(

Ta5d6fffTa5d6ff"Tffea00\nTa5d6ffRunning Tffd700{Te6edf3num_threadsTffd700}Ta5d6ff threads inserting Tffd700{Te6edf3announces_per_threadTffd700}Ta5d6ff announces each...Ta5d6ff"Tb4b4b4,

Tb4b4b4)

Te6edf3start Tff7b72= Te6edf3timeTff7b72.Td2a8fftimeTb4b4b4(Tb4b4b4)

Tff7b72for Te6edf3t Tff7b72in Te6edf3threadsTb4b4b4:

Te6edf3tTff7b72.Td2a8ffstartTb4b4b4(Tb4b4b4)

Tff7b72for Te6edf3t Tff7b72in Te6edf3threadsTb4b4b4:

Te6edf3tTff7b72.Td2a8ffjoinTb4b4b4(Tb4b4b4)

Te6edf3duration Tff7b72= Te6edf3timeTff7b72.Td2a8fftimeTb4b4b4(Tb4b4b4) Tff7b72- Te6edf3start

Tffa657printTb4b4b4(

Ta5d6fffTa5d6ff"Ta5d6ffConcurrent insertion took Tffd700{Te6edf3durationTffd700:Ta5d6ff.2fTffd700}Ta5d6ffs for Tffd700{Te6edf3num_threads Tff7b72* Te6edf3announces_per_threadTffd700}Ta5d6ff announcesTa5d6ff"Tb4b4b4,

Tb4b4b4)

Tff7b72selfTff7b72.Td2a8ffassertLessTb4b4b4(Te6edf3durationTb4b4b4, T79c0ff10.0Tb4b4b4, Ta5d6ff"Ta5d6ffConcurrent announce insertion is too slow!Ta5d6ff"Tb4b4b4)

Tff7b72if Tff7b72__name__ Tff7b72== Ta5d6ff"Ta5d6ff__main__Ta5d6ff"Tb4b4b4:

Te6edf3unittestTff7b72.Td2a8ffmainTb4b4b4(Tb4b4b4)